# Find unique values
unique_values = df_3['Sentiment'].unique()
unique_values

# Find unique values
unique_values = df_3['Sentiment'].unique()
unique_values

# Check data types
df_3.dtypes

df_3

test_data_y

# Find average number of tokens (words) in training Tweets
round(sum([len(i.split()) for i in test_data_x])/len(test_data_x))

# Test text vecotrizer 
sample_sentence = "I hate covid a lot and it sucks"
text_vectorizer([sample_sentence])

# Read a random tweet from training data and vectorize it
import random
random_sentence = random.choice(train_data_x)
print(f"Original text:\n{random_sentence}\
      \n\nVectorized version:")
text_vectorizer([random_sentence])

# Get the unique words in the vocabulary
words_in_vocab = text_vectorizer.get_vocabulary()
top_5_words = words_in_vocab[:5] # most common tokens (notice the [UNK] token for "unknown" words)
bottom_5_words = words_in_vocab[-5:] # least common tokens
print(f"Number of words in vocab: {len(words_in_vocab)}")
print(f"Top 5 most common words: {top_5_words}") 
print(f"Bottom 5 least common words: {bottom_5_words}")

# Create an embedding layer for neural network
tf.random.set_seed(42)
from tensorflow.keras import layers

embedding = layers.Embedding(input_dim=max_vocab_length, # set input shape
                             output_dim=128, # set size of embedding vector
                             embeddings_initializer="uniform", # default, intialize randomly
                             input_length=max_length, # how long is each input
                             name="embedding_1") 

embedding

# Get a random sentence from training set
random_sentence = random.choice(train_data_x)
print(f"Original text:\n{random_sentence}\
      \n\nEmbedded version:")

# Embed the random sentence (turn it into numerical representation)
sample_embed = embedding(text_vectorizer([random_sentence]))
sample_embed,sample_embed[0][0]

# Use a Multinomial Naive Bayes Model as a baseline
# TfidfVectorizer() is a Feature extraction technique that converts the text data into numerical features using 
# the Term Frequency-Inverse Document Frequency (TF-IDF) 
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline

# Create tokenization and modelling pipeline
model_0 = Pipeline([
                    ("tfidf", TfidfVectorizer()), # convert words to numbers using tfidf
                    ("clf", MultinomialNB()) # model the text
])

# Fit the pipeline to the training data
model_0.fit(train_data_x, train_data_y)

# Check results
model_0_results = calculate_results(y_true=test_data_y, y_pred=model_0.predict(test_data_x))
model_0_results 

# Use a tensorflow simple dense model
from tensorflow.keras import layers
inputs = layers.Input(shape=(1,), dtype="string") # inputs are 1-dimensional strings
x = text_vectorizer(inputs) # turn the input text into numbers
x = embedding(x) # create an embedding of the numerized numbers
x = layers.GlobalAveragePooling1D()(x) # lower the dimensionality of the embedding (try running the model without this layer and see what happens)
outputs = layers.Dense(5, activation="softmax")(x) 
model_1 = tf.keras.Model(inputs, outputs, name="model_1_dense") 

model_1.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(),
                optimizer=tf.keras.optimizers.Adam(),
                metrics=["accuracy"])

history_1 = model_1.fit(train_data_x, # input sentences can be a list of strings due to text preprocessing layer built-in model
                              train_data_y,
                              epochs=5,
                              validation_data=(test_data_x, test_data_y))

# Use a LSTM model
tf.random.set_seed(42)
from tensorflow.keras import layers
model_2_embedding = layers.Embedding(input_dim=max_vocab_length,
                                     output_dim=128,
                                     embeddings_initializer="uniform",
                                     input_length=max_length,
                                     name="embedding_2")

# Create LSTM model
inputs = layers.Input(shape=(1,), dtype="string")
x = text_vectorizer(inputs)
x = model_2_embedding(x)
print(x.shape)
# x = layers.LSTM(64, return_sequences=True)(x) # return vector for each word in the Tweet (you can stack RNN cells as long as return_sequences=True)
x = layers.LSTM(64)(x) # return vector for whole sequence
print(x.shape)
# x = layers.Dense(64, activation="relu")(x) # optional dense layer on top of output of LSTM cell
outputs = layers.Dense(5, activation="softmax")(x)
model_2 = tf.keras.Model(inputs, outputs, name="model_2_LSTM")

model_2.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(),
                optimizer=tf.keras.optimizers.Adam(),
                metrics=["accuracy"])

history_2 = model_2.fit(train_data_x, # input sentences can be a list of strings due to text preprocessing layer built-in model
                              train_data_y,
                              epochs=5,
                              validation_data=(test_data_x, test_data_y))

# Use a GRU model
tf.random.set_seed(42)
from tensorflow.keras import layers
model_3_embedding = layers.Embedding(input_dim=max_vocab_length,
                                     output_dim=128,
                                     embeddings_initializer="uniform",
                                     input_length=max_length,
                                     name="embedding_3")

# Build an RNN using the GRU cell
inputs = layers.Input(shape=(1,), dtype="string")
x = text_vectorizer(inputs)
x = model_3_embedding(x)
# x = layers.GRU(64, return_sequences=True) # stacking recurrent cells requires return_sequences=True
x = layers.GRU(64)(x) 
# x = layers.Dense(64, activation="relu")(x) # optional dense layer after GRU cell
outputs = layers.Dense(5, activation="softmax")(x)
model_3 = tf.keras.Model(inputs, outputs, name="model_3_GRU")


model_3.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(),
                optimizer=tf.keras.optimizers.Adam(),
                metrics=["accuracy"])

# Fit model
history_3 = model_3.fit(train_data_x, # input sentences can be a list of strings due to text preprocessing layer built-in model
                              train_data_y,
                              epochs=5,
                              validation_data=(test_data_x, test_data_y))

# Use a bidirectional RNN model
tf.random.set_seed(42)
from tensorflow.keras import layers
model_4_embedding = layers.Embedding(input_dim=max_vocab_length,
                                     output_dim=128,
                                     embeddings_initializer="uniform",
                                     input_length=max_length,
                                     name="embedding_4")

# Build a Bidirectional RNN in TensorFlow
inputs = layers.Input(shape=(1,), dtype="string")
x = text_vectorizer(inputs)
x = model_4_embedding(x)
# x = layers.Bidirectional(layers.LSTM(64, return_sequences=True))(x) # stacking RNN layers requires return_sequences=True
x = layers.Bidirectional(layers.LSTM(64))(x) # bidirectional goes both ways so has double the parameters of a regular LSTM layer
outputs = layers.Dense(5, activation="softmax")(x)
model_4 = tf.keras.Model(inputs, outputs, name="model_4_Bidirectional")
# Compile
model_4.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(),
                optimizer=tf.keras.optimizers.Adam(),
                metrics=["accuracy"])

# Fit the model (takes longer because of the bidirectional layers)
history_4 = model_4.fit(train_data_x, # input sentences can be a list of strings due to text preprocessing layer built-in model
                              train_data_y,
                              epochs=5,
                              validation_data=(test_data_x, test_data_y))

# Use a 1-d convolutional layer model
tf.random.set_seed(42)
from tensorflow.keras import layers
model_5_embedding = layers.Embedding(input_dim=max_vocab_length,
                                     output_dim=128,
                                     embeddings_initializer="uniform",
                                     input_length=max_length,
                                     name="embedding_5")

# Create 1-dimensional convolutional layer to model sequences
from tensorflow.keras import layers
inputs = layers.Input(shape=(1,), dtype="string")
x = text_vectorizer(inputs)
x = model_5_embedding(x)
x = layers.Conv1D(filters=32, kernel_size=5, activation="relu")(x)
x = layers.GlobalMaxPool1D()(x)
# x = layers.Dense(64, activation="relu")(x) # optional dense layer
outputs = layers.Dense(5, activation="softmax")(x)
model_5 = tf.keras.Model(inputs, outputs, name="model_5_Conv1D")

# Compile Conv1D model
model_5.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(),
                optimizer=tf.keras.optimizers.Adam(),
                metrics=["accuracy"])
history_5 =  model_5.fit(train_data_x, # input sentences can be a list of strings due to text preprocessing layer built-in model
                              train_data_y,
                              epochs=5,
                              validation_data=(test_data_x, test_data_y))

# Use a transfer learning model with sentence encoder
import tensorflow_hub as hub
sentence_encoder_layer = hub.KerasLayer("https://tfhub.dev/google/universal-sentence-encoder/4",
                                        input_shape=[], # shape of inputs coming to our model 
                                        dtype=tf.string, # data type of inputs coming to the USE layer
                                        trainable=False, # keep the pretrained weights (we'll create a feature extractor)
                                        name="USE") 

# Create model using the Sequential API
model_6 = tf.keras.Sequential([
  sentence_encoder_layer, # take in sentences and then encode them into an embedding
  layers.Dense(64, activation="relu"),
  layers.Dense(5, activation="softmax")
], name="model_6_USE")

# Compile model
model_6.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(),
                optimizer=tf.keras.optimizers.Adam(),
                metrics=["accuracy"])

# Train a classifier on top of pretrained embeddings
history_6 = model_6.fit(train_data_x, # input sentences can be a list of strings due to text preprocessing layer built-in model
                              train_data_y,
                              epochs=5,
                              validation_data=(test_data_x, test_data_y))

model_0_results = calculate_results(y_true=test_data_y, y_pred=model_0.predict(test_data_x))
model_1_results = calculate_results(y_true=test_data_y, y_pred=model_1.predict(test_data_x).argmax(axis=1))
model_2_results = calculate_results(y_true=test_data_y, y_pred=model_2.predict(test_data_x).argmax(axis=1))
model_3_results = calculate_results(y_true=test_data_y, y_pred=model_3.predict(test_data_x).argmax(axis=1))
model_4_results = calculate_results(y_true=test_data_y, y_pred=model_4.predict(test_data_x).argmax(axis=1))
model_5_results = calculate_results(y_true=test_data_y, y_pred=model_5.predict(test_data_x).argmax(axis=1))
model_6_results = calculate_results(y_true=test_data_y, y_pred=model_6.predict(test_data_x).argmax(axis=1))
ensemble_results = calculate_results(y_true=test_data_y,  y_pred=((model_2.predict(test_data_x) + model_3.predict(test_data_x) + model_4.predict(test_data_x))/3).argmax(axis=1))

# Combine model results into a DataFrame
all_model_results = pd.DataFrame({"naive bayes": model_0_results,
                                  "simple_dense": model_1_results,
                                  "lstm": model_2_results,
                                  "gru": model_3_results,
                                  "bidirectional": model_4_results,
                                  "conv1d": model_5_results,
                                  "tf_hub_sentence_encoder": model_6_results,
                                   "ensemble_results":ensemble_results})
all_model_results = all_model_results.transpose()
# Reduce the accuracy to same scale as other metrics
all_model_results["accuracy"] = all_model_results["accuracy"]/100
all_model_results

# Plot and compare all of the model results
all_model_results.plot(kind="bar", figsize=(10, 7)).legend(bbox_to_anchor=(1.0, 1.0));

# Sort model results by f1-score
all_model_results.sort_values("f1", ascending=False)["f1"].plot(kind="bar", figsize=(10, 7));

# Want to get weights of embedding layer for LSTM model
model_2.summary()

# Calculate prediction times for all models
model_0_total_pred_time, model_0_time_per_pred = pred_timer(model_0, test_data_x)
model_1_total_pred_time, model_1_time_per_pred = pred_timer(model_1, test_data_x)
model_2_total_pred_time, model_2_time_per_pred = pred_timer(model_2, test_data_x)
model_3_total_pred_time, model_3_time_per_pred = pred_timer(model_3, test_data_x)
model_4_total_pred_time, model_4_time_per_pred = pred_timer(model_4, test_data_x)
model_5_total_pred_time, model_5_time_per_pred = pred_timer(model_5, test_data_x)
model_6_total_pred_time, model_6_time_per_pred = pred_timer(model_6, test_data_x)

# Make Scatter plot
import matplotlib.pyplot as plt
plt.figure(figsize=(10, 7))
plt.scatter(model_0_time_per_pred, model_0_results["f1"], label="naive bayes")
plt.scatter(model_1_time_per_pred, model_1_results["f1"], label="simple_dense")
plt.scatter(model_2_time_per_pred, model_2_results["f1"], label="lstm")
plt.scatter(model_3_time_per_pred, model_3_results["f1"], label="gru")
plt.scatter(model_4_time_per_pred, model_4_results["f1"], label="bidirectional")
plt.scatter(model_5_time_per_pred, model_5_results["f1"], label="conv1d")
plt.scatter(model_6_time_per_pred, model_6_results["f1"], label="tf_hub_sentence_encoder")
plt.legend()
plt.title("F1-score versus time per prediction")
plt.xlabel("Time per prediction")
plt.ylabel("F1-Score");